# Torch
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import matplotlib.pyplot as plt
from torch.utils.data import Dataset, DataLoader, Subset, random_split
from tqdm import tqdm

# Augmentation
import albumentations as A
from albumentations.pytorch import ToTensorV2
import scipy.io as sio

# Visualize Result
from sklearn.metrics import (confusion_matrix, accuracy_score,
                            precision_score, recall_score,
                            f1_score, roc_auc_score,
                            roc_curve, auc, precision_recall_curve,
                            average_precision_score)
from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.preprocessing import label_binarize

# Basic
import numpy as np
import cv2
import os
import time
from typing import List, Tuple, Union
import random
import itertools
import copy

E:\Courses\CISC3024-Pattern-Recognition\cisc3024_pr_venv\lib\site-packages\albumentations\__init__.py:13: UserWarning: A new version of Albumentations is available: 1.4.21 (you have 1.4.18). Upgrade using: pip install -U albumentations. To disable automatic update checks, set the environment variable NO_ALBUMENTATIONS_UPDATE to 1.
  check_for_updates()

device_name = "cuda" if torch.cuda.is_available() else "cpu"
device = torch.device(device_name)
print(f"Using device: {device_name}")

Using device: cuda

path_dataset = "./data/SVHN_mat"

_dat = sio.loadmat(os.path.join(path_dataset, "train_32x32.mat"))
# _dat['X'][0][0][0]
# np.array(_dat).shape
dat = np.transpose(_dat['X'], (3, 0, 1, 2))
# dat = dat.astype(np.float32)
dat.shape

# First Image
dat[0].shape

# First Row of Image
dat[0][0]

# First Pixel of Image
dat[0][0][0]

_transform = A.Compose([
    A.Normalize(mean=[0.4376845359802246, 0.4437684714794159, 0.47280389070510864], std=[0.19803018867969513, 0.2010156661272049, 0.19703581929206848]),
    ToTensorV2()
])

_img = dat[0]
_img = _transform(image=_img)['image']
# print(_img)
_img.shape

class SVHNDataset(Dataset):
    def __init__(self, mat_file, transform=None):
        data = sio.loadmat(mat_file)
        
        self.images = np.transpose(data['X'], (3, 0, 1, 2))
        self.labels = data['y'].flatten()
        self.labels[self.labels == 10] = 0
        self.transform = transform        # Allow postponed injection of transform.

    def __len__(self):
        return len(self.labels)

    def __getitem__(self, idx):
        image = self.images[idx]
        label = self.labels[idx]

        # There should always be a transform. 
        # It converts image to float, and permutes it from (32, 32, 3) to Tensor([3, 32, 32]).
        # ...which is important!!
        if self.transform is None:        
            raise ValueError("CISC3024 Custom Error: The transform should not be None when this object is passed into a DataLoader.")

        image = self.transform(image=image)['image']
        return image, label
    
    def get_meanstd(self, bias=None):
        if bias is not None:
            random_bias = random.randint(0, bias)
            images_ = []
            for i in range(len(self.images)):
                image = self.images[i]
                image = image.astype(np.int16)
                image = (image + random_bias) % 256
                image = image.astype(np.uint8)
                images_.append(image)
            images_ = np.array(images_)
        else:
            images_ = self.images
                
        images_ = images_.astype(np.float32) / 255.0
        mean = np.mean(images_, axis=(0,1,2))
        std = np.std(images_, axis=(0,1,2), ddof=0)
        
        return mean.tolist(), std.tolist()        
    
    def overwrite(self, indices:Union[list, np.ndarray]):
        """
        Create a deep copy of the mother dataset instance and only keep the wanted
        data samples, controlled by indices.
        """
        if any(index < 0 or index >= len(self.labels) for index in indices):
            raise IndexError("CISC3024 Custom Error: One or more indices are out of bounds.")
        
        new_dataset = copy.deepcopy(self)
        new_dataset.images = self.images[indices]
        new_dataset.labels = self.labels[indices]
        return new_dataset

def peek(dataset, index=None):
    def unnormalize(img, mean, std):
        """Revert the normalization for visualization."""
        img = img * std + mean
        return np.clip(img, 0, 1)

    mean, std = dataset.get_meanstd()

    # Plotting multiple images in a grid
    grid_rows, grid_cols = 1, 6
    
    fig, axes = plt.subplots(grid_rows, grid_cols, figsize=(6, 6))

    peek_index = random.randint(0, dataset.__len__()-1) if index is None else index
    
    for i in range(grid_cols):
        img_tensor, label = dataset.__getitem__(peek_index)
        img = img_tensor.permute(1, 2, 0).numpy()  # Convert to (H, W, C)
        img = unnormalize(img, mean, std)
    
        ax = axes[i]  # Get subplot axis
        ax.imshow(img)
        ax.set_title(f"Label: {label}")
    
    plt.tight_layout()
    plt.show()
    print(f"Peeking data from training set of index {peek_index}.\nImage Tnesor Size:{dataset.__getitem__(peek_index)[0].shape}")

class SmallVGG(nn.Module):
    def __init__(self, frame_size=32):
        super(SmallVGG, self).__init__()
        self.frame_size = frame_size
        self.conv_layers = nn.Sequential(
            nn.Conv2d(3, 8, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Conv2d(8, 16, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2), # 16x16

            nn.Conv2d(16, 32, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Conv2d(32, 32, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2), # 8x8

            nn.Conv2d(32, 32, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Conv2d(32, 32, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2), # 4x4
        )
    
        self.fc_layers = nn.Sequential(
            nn.Linear(frame_size * 4 * 4, 256),
            nn.ReLU(),
            nn.Linear(256, 10)
        )

    def forward(self, x):
        x = self.conv_layers(x)
        x = x.view(x.size(0), -1)
        x = self.fc_layers(x)
        return x

def train_and_evaluate(model,
                      train_loader,
                      valid_loader,
                      criterion,
                      optimizer,
                      num_epochs=100,
                      stop_early_params=None):
    # Record Losses to plot
    train_losses = []
    valid_losses = []

    # Early stop params
    current_optimized_model = None
    current_min_valid_loss = np.inf
    num_overfit_epochs = 0

    for epoch in range(num_epochs):
        # Train
        model.train()
        running_loss = 0.0
        for images, labels in tqdm(train_loader):
            images, labels = images.to(device), labels.to(device)
            optimizer.zero_grad()
            outputs = model(images)

            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()

            running_loss += loss.item() * len(images)
        train_losses.append(running_loss / len(train_loader))

        # Evaluate
        model.eval()
        valid_loss = 0.0
        with torch.no_grad():
            for images, labels in valid_loader:
                images, labels = images.to(device), labels.to(device)
                outputs = model(images)
                loss = criterion(outputs, labels)
                valid_loss += loss.item() * len(images)

        valid_losses.append(valid_loss / len(valid_loader))
        print(f"Epoch[{epoch+1}/{num_epochs}], Train Loss:{train_losses[-1]:.4f}, Validation Loss:{valid_losses[-1]:.4f}")

        # Early Stop?
        if stop_early_params is None:
            continue
    
        if current_min_valid_loss - stop_early_params["min_delta"] > valid_losses[-1]: # Validation loss decreases
            current_min_valid_loss = valid_losses[-1]
            current_optimized_model = copy.deepcopy(model)
            num_overfit_epochs = (num_overfit_epochs - 1) if num_overfit_epochs > 0 else 0
        else: # Validation loss increases
            num_overfit_epochs += 1

        if num_overfit_epochs > stop_early_params["patience"]:
            print(f"Early stopping at epoch {epoch+1}.")
            model = current_optimized_model
            break

    return train_losses, valid_losses

def get_predictions(model_path, extra_loader):
    if not isinstance(model_path, str):
        model_state = model_path
    else:
        model_state = torch.load(model_path)
    model = SmallVGG()
    model.load_state_dict(model_state)
    
    model.to(device)
    model.eval() 
    
    pred_scores = []  # Prob. of predictions
    true_labels = []  # Ground Truth
    pred_labels = []  # Label of prediction, i.e., argmax(softmax(pred_scores))
    
    with torch.no_grad():
        for images, labels in tqdm(extra_loader):
            images, labels = images.to(device), labels.to(device)
        
            outputs = model(images)
    
            pred_scores_batch = nn.functional.softmax(outputs, dim=-1)
    
            pred_scores.extend(pred_scores_batch.cpu().tolist())
            pred_labels.extend(outputs.argmax(dim=1).tolist())
            true_labels.extend(labels.cpu().tolist())
            
    return pred_scores, true_labels, pred_labels

def get_metrics(true_labels, pred_labels):
    accuracy = accuracy_score(true_labels, pred_labels)
    precision = precision_score(true_labels, pred_labels, zero_division=1, average=None, labels=range(0,10))
    recall = recall_score(true_labels, pred_labels, zero_division=1, average=None, labels=range(0,10))
    f1 = f1_score(true_labels, pred_labels, zero_division=0, average=None, labels=range(0,10))

    return accuracy, precision, recall, f1

def print_metrics(accuracies, f1s):
    print(f"Accuracies:")
    for acc in accuracies:
        print(f"{acc:.3f}", end=" ")
    print("\n")
    
    print(f"F1 Score Lists:")
    mean_f1s = []
    for f1 in f1s:
        for val in f1:
            print(f"{val:.3f}", end=" ")
        mean_f1 = np.mean(f1)
        std_f1 = np.std(f1)
        mean_f1s.append(mean_f1)
        print(f"| Avg F1={mean_f1:.3f}, Std F1={std_f1}")
    print(f"Best: {np.argmax(mean_f1s)+1}-th")

# Compute ROC AUC for each class
def get_roc_auc(true_labels_bin, pred_labels_bin):
    roc_auc = dict()
    for i in range(0, 10):
        roc_auc[i] = roc_auc_score(true_labels_bin[:,i], np.array(pred_scores)[:, i])
    return roc_auc

{
    "HYPER_PARAM_1": combo[0],
    "HYPER_PARAM_2": combo[1],
    "train_losses": train_losses,
    "valid_losses": valid_losses,
    "model_state_dict": exp_model.state_dict()
}

def plot_el(loaded_experiments, hyper_param_names, n_rows=4, n_cols=4):
    fig_size = (n_cols * 5, n_rows * 5)
    n1, n2 = hyper_param_names
    
    fig, axes = plt.subplots(nrows=n_rows, ncols=n_cols, figsize=fig_size)
    # plt.tight_layout()
    
    for i, ax in enumerate(axes.flat):
        train_losses, valid_losses = loaded_experiments[i]["train_losses"], loaded_experiments[i]["valid_losses"]
        
        ax.plot(train_losses, label=f"TRL, min={np.min(train_losses):.3f}")
        ax.plot(valid_losses, label=f"VAL, min={np.min(valid_losses):.3f} at step={np.argmin(valid_losses)}")
        ax.set_xlabel("Epochs")
        ax.set_ylabel("Loss")
        ax.set_title(f"{n1}={loaded_experiments[i][n1]}, {n2}={loaded_experiments[i][n2]}")
        ax.legend(loc="upper right")
    
    plt.show()

def get_experiment_results(loaded_experiments, test_hyperparam_names, extra_loader):
    experiment_results = []
    n1, n2 = test_hyperparam_names
    for i, exp in enumerate(loaded_experiments):
        pred_scores, true_labels, pred_labels = get_predictions(exp['model_state_dict'], extra_loader)
        experiment_results.append({
            n1: exp[n1],
            n2: exp[n2],
            "true_labels": true_labels,
            "pred_labels": pred_labels,
            "pred_scores": pred_scores
        })

        print(f"First 10 true labels:")
        [print(num, end=" ") for num in true_labels[:10]]
        print(f"...\n")

        print(f"First 10 pred labels:")
        [print(num, end=" ") for num in pred_labels[:10]]
        print(f"...\n")

        print(f"First 5 pred_scores:")
        [print(num, end=" ") for num in pred_scores[:5]]
        print(f"...\n")

        # del pred_scores, true_labels, pred_lables
        torch.cuda.empty_cache()
    return experiment_results

def plot_cm(experiment_results, hyper_param_names, n_rows=4, n_cols=4):
    fig, axes = plt.subplots(n_rows, n_cols, figsize=(n_cols * 5, n_rows * 5))
    axes = axes.flatten()

    hparam_1, hparam_2 = hyper_param_names
    
    for i, exp_rs in enumerate(experiment_results):
        true_labels, pred_labels = exp_rs['true_labels'], exp_rs['pred_labels']
        cm = confusion_matrix(true_labels, pred_labels)
        disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=range(0,10))
        disp.plot(ax=axes[i], cmap = plt.cm.Blues)
        axes[i].set_title(f"Exp {i+1}: {hparam_1}={exp_rs[hparam_1]}, {hparam_2}={exp_rs[hparam_2]}")
    
    plt.tight_layout()
    plt.show()

def plot_pr(experiment_results, hyper_param_names, n_rows=4, n_cols=4):
    fig, axes = plt.subplots(n_rows,n_cols, figsize=(n_cols * 5, n_rows * 5))
    axes = axes.flatten()

    hparam_1, hparam_2 = hyper_param_names
    accuracies = []
    f1_scores = []
    
    for i, exp_rs in enumerate(experiment_results):
        true_labels, pred_labels, pred_scores = exp_rs['true_labels'], exp_rs['pred_labels'], exp_rs['pred_scores']
        true_labels_bin, pred_labels_bin = label_binarize(true_labels, classes=range(0,10)), label_binarize(pred_labels, classes=range(0,10))
        
        accuracy, precision, recall, f1 = get_metrics(true_labels, pred_labels)
        accuracies.append(accuracy)
        f1_scores.append(f1)
        
        for j in range(0, 10):
            # print(f"Class {j}: Prec:{precision[j]:.2f}, Recall:{recall[j]:.2f}, F_1 Score:{f1[j]:.2f}")
            precision_i, recall_i, _ = precision_recall_curve(true_labels_bin[:, j], np.array(pred_scores)[:, j])
    
            average_precision = average_precision_score(true_labels_bin[:, j], np.array(pred_scores)[:, j])
            axes[i].step(recall_i, precision_i, where="post", label=f"Class {j} AP={average_precision:.2f}")
            axes[i].set_title(f"PR-Curve {hparam_1}={exp_rs[hparam_1]}, {hparam_2}={exp_rs[hparam_2]}")
        axes[i].legend()
        axes[i].set_xlabel("Recall")
        axes[i].set_ylabel("Precision")
    
    # for j in range(i+1, 16):
    #     fig.delaxes(axes[j])
    
    plt.tight_layout()
    plt.show()
    return accuracies, f1_scores

def plot_rocauc(experiment_results, hyper_param_names, curve_type, n_rows=4, n_cols=4):
    fig, axes = plt.subplots(n_rows, n_cols, figsize=(n_cols * 5, n_rows * 5))
    axes = axes.flatten()

    hparam_1, hparam_2 = hyper_param_names

    for i, exp_rs in enumerate(experiment_results):
        true_labels, pred_scores = exp_rs['true_labels'], exp_rs['pred_scores']
        true_labels_bin = label_binarize(true_labels, classes=range(0, 10))

        # All Classes' ROC curve & ROC Area Under Curve
        fpr = dict()
        tpr = dict()
        roc_auc = dict()

        for j in range(10):
            fpr[j], tpr[j], _ = roc_curve(true_labels_bin[:, j], np.array(pred_scores)[:, j])
            roc_auc[j] = auc(fpr[j], tpr[j])

        # Macro-Average ROC & ROC-AUC
        all_fpr = np.unique(np.concatenate([fpr[j] for j in range(10)]))
        mean_tpr = np.zeros_like(all_fpr)
        for j in range(10):
            mean_tpr += np.interp(all_fpr, fpr[j], tpr[j])
        mean_tpr /= 10

        fpr["macro"] = all_fpr
        tpr["macro"] = mean_tpr
        roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])

        # Compute micro-average ROC curve and ROC area
        fpr["micro"], tpr["micro"], _ = roc_curve(true_labels_bin.ravel(), np.array(pred_scores).ravel())
        roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])

        # Plot only Macro or Micro ROC curves
        if curve_type == "macro_micro":
            axes[i].plot(fpr["macro"], tpr["macro"], label=f"Macro (AUC={roc_auc['macro']:.2f})")
            axes[i].plot(fpr["micro"], tpr["micro"], label=f"Micro (AUC={roc_auc['micro']:.2f})")
        elif curve_type == "all":
            # Plot all ROC curves
            for j in range(10):
                axes[i].plot(fpr[j], tpr[j], label=f"Class {j} (AUC={roc_auc[j]:.2f})")

        axes[i].plot([0, 1], [0, 1], "k--")
        axes[i].set_xlabel("False Positive Rate")
        axes[i].set_ylabel("True Positive Rate")
        axes[i].set_title(f"ROC Curve {i+1}, {hparam_1}={exp_rs[hparam_1]}, {hparam_2}={exp_rs[hparam_2]}")
        axes[i].legend(loc='lower right')

    plt.tight_layout()
    plt.show()

def split_train_valid(train_dataset, train_ratio):
    ori_len = len(train_dataset)
    train_size = int(train_ratio * ori_len)
    valid_size = ori_len - train_size

    # These are subsets!! Don't directly use them or you will spend 2 hours solving for it.
    train_subset, valid_subset = random_split(train_dataset, [train_size, valid_size])

    # Re-construct two SVHNDataset object from indecies    
    train_dataset_ = train_dataset.overwrite(indices=train_subset.indices)
    valid_dataset_ = train_dataset.overwrite(indices=valid_subset.indices)
    
    return train_dataset_, valid_dataset_

# Universal Train Dataset without splitting
exp1_universal_train_dataset = SVHNDataset(mat_file=os.path.join(path_dataset,"train_32x32.mat"))

# Train-Validation Split
exp1_train_dataset, exp1_valid_dataset = split_train_valid(exp1_universal_train_dataset, train_ratio=0.8)

if not isinstance(exp1_train_dataset, SVHNDataset) or not isinstance(exp1_valid_dataset, SVHNDataset):
    raise TypeError("CISC3024 Custom Error: The dataset should be an instance of SVHNDataset.")

# Normalize
exp1_mean, exp1_std = exp1_train_dataset.get_meanstd()
exp1_hyperparams = {
    "num_epochs": 25,
    "lr": 1e-3,
    "criterion": nn.CrossEntropyLoss(),
    "transform": A.Compose([
        A.Normalize(mean=exp1_mean, std=exp1_std),
        ToTensorV2()
    ])
}

# Inject Transform
exp1_train_dataset.transform = exp1_hyperparams['transform']
exp1_valid_dataset.transform = exp1_hyperparams['transform']

# Test Dataset
exp1_test_dataset = SVHNDataset(mat_file=os.path.join(path_dataset, "test_32x32.mat"), transform=exp1_hyperparams['transform']) 

del exp1_universal_train_dataset

print(f"Training Size:{exp1_train_dataset.__len__()}, Validation Size:{exp1_valid_dataset.__len__()}")
print(f"Channel Means: {exp1_mean}\nChannel Stds: {exp1_std}")

Training Size:58605, Validation Size:14652
Channel Means: [0.43772128224372864, 0.44378969073295593, 0.4728474020957947]
Channel Stds: [0.19793079793453217, 0.20086902379989624, 0.1968136429786682]

exp1_models = [SmallVGG().to(device) for _ in range(0,6)]

candidate_optimizers = [
    optim.Adam(exp1_models[0].parameters(), lr=exp1_hyperparams['lr']), 
    optim.SGD(exp1_models[1].parameters(), lr=exp1_hyperparams['lr'], momentum=0.9),
    optim.RMSprop(exp1_models[2].parameters(), lr=exp1_hyperparams['lr']),
    optim.AdamW(exp1_models[3].parameters(), lr=exp1_hyperparams['lr'], weight_decay=0.01),
    optim.Adagrad(exp1_models[4].parameters(), lr=exp1_hyperparams['lr']),
    optim.SGD(exp1_models[5].parameters(), lr=exp1_hyperparams['lr'], momentum=0.9, nesterov=True)]

for model in exp1_models:
    print(id(model), end=", ")

3044296310112, 3043927082032, 3043927083520, 3043927084144, 3043927085008, 3044137279152,

# Data Loaders
exp1_train_loader = DataLoader(exp1_train_dataset, batch_size=128, shuffle=True)
exp1_valid_loader = DataLoader(exp1_valid_dataset, batch_size=128, shuffle=True)
exp1_test_loader = DataLoader(exp1_test_dataset, batch_size=128, shuffle=False)

def run_exp1(optimizers, models, hyper_params, train_loader, valid_loader):
    experiments = []
    for i, [optimizer, exp1_model] in enumerate(zip(optimizers, models)):
        print(f"Experiment {i+1}. Running experiment on optimizer: {optimizer.__class__.__name__}")

        criterion = hyper_params['criterion']
        num_epochs = hyper_params['num_epochs']
        train_losses, valid_losses = train_and_evaluate(exp1_model, train_loader, valid_loader, criterion, optimizer, num_epochs)

        experiments.append({
            "optimizer": optimizer.__class__.__name__,
            "others":"same",
            "train_losses": train_losses,
            "valid_losses": valid_losses,
            "model_state_dict": exp1_model.state_dict()
        })

        del exp1_model, criterion, optimizer
        torch.cuda.empty_cache()

    return experiments

exp1 = run_exp1(candidate_optimizers, exp1_models, exp1_hyperparams, exp1_train_loader, exp1_valid_loader)
time_str = str(time.time()).replace(".","")
torch.save(exp1, f"./models/exp1_{time_str}.pth")

exp1_loaded = torch.load("./models/exp1_17305518422052872.pth")
exp1_results = get_experiment_results(exp1_loaded, test_hyperparam_names=["optimizer", "others"], extra_loader=exp1_test_loader)

plot_el(exp1_loaded, ["optimizer", "others"], n_rows=1, n_cols=6)

plot_cm(exp1_results, ["optimizer", "others"], n_rows=1, n_cols=6)

exp1_accuracies, exp1_f1s = plot_pr(exp1_results, ["optimizer", "others"], n_rows=1, n_cols=6)
print_metrics(exp1_accuracies, exp1_f1s)

Accuracies:
0.907 0.196 0.906 0.901 0.735 0.196 

F1 Score Lists:
0.912 0.935 0.938 0.874 0.921 0.905 0.881 0.906 0.847 0.867 | Avg F1=0.899, Std F1=0.028664446684648247
0.000 0.328 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 | Avg F1=0.033, Std F1=0.09827503131926378
0.888 0.933 0.939 0.876 0.913 0.903 0.878 0.923 0.850 0.865 | Avg F1=0.897, Std F1=0.028470034992916605
0.887 0.935 0.936 0.871 0.914 0.907 0.871 0.908 0.830 0.843 | Avg F1=0.890, Std F1=0.03434799781092271
0.723 0.840 0.802 0.638 0.776 0.700 0.647 0.789 0.515 0.628 | Avg F1=0.706, Std F1=0.09492988307421246
0.000 0.328 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 | Avg F1=0.033, Std F1=0.09827503131926378
Best: 1-th

plot_rocauc(exp1_results, ["optimizer", "others"], curve_type="all", n_rows=1, n_cols=6)

plot_rocauc(exp1_results, ["optimizer", "others"], curve_type="macro_micro", n_rows=1, n_cols=6)

# Universal Train Dataset without splitting
exp2_universal_train_dataset = SVHNDataset(mat_file=os.path.join(path_dataset,"train_32x32.mat"))

# Train & Validation Datasets
exp2_train_dataset, exp2_valid_dataset = split_train_valid(exp2_universal_train_dataset, train_ratio=0.8)
del exp2_universal_train_dataset # Unload the mill and kill the donkey

# Normalize
exp2_mean, exp2_std = exp2_train_dataset.get_meanstd()

exp2_hyperparams = {
    "criterion": nn.CrossEntropyLoss(),
    "transform": A.Compose([
        A.Normalize(mean=exp2_mean, std=exp2_std),
        ToTensorV2()
    ]),
    "optimizer":optim.Adam,
}

exp2_train_dataset.transform = exp2_hyperparams['transform']
exp2_valid_dataset.transform = exp2_hyperparams['transform']

# Test Dataset
exp2_test_dataset = SVHNDataset(mat_file=os.path.join(path_dataset,"test_32x32.mat"), transform=exp2_hyperparams['transform'])

print(f"Training Size:{exp2_train_dataset.__len__()}, Validation Size:{exp2_valid_dataset.__len__()}")
print(f"Channel Means: {exp2_mean}\nChannel Stds: {exp2_std}")

Training Size:58605, Validation Size:14652
Channel Means: [0.4374935030937195, 0.44353100657463074, 0.4726291298866272]
Channel Stds: [0.1980431079864502, 0.2009742110967636, 0.19704842567443848]

candidate_epochs = [10, 15, 20, 25]
candidate_lr = [1e-3, 1e-4, 1e-5, 1e-6]

exp2_train_loader = DataLoader(exp2_train_dataset, batch_size=128, shuffle=True)
exp2_valid_loader = DataLoader(exp2_valid_dataset, batch_size=128, shuffle=True)
exp2_test_loader = DataLoader(exp2_test_dataset, batch_size=128, shuffle=False)

def run_exp2_1(epochs, lr_list, hyper_params, train_loader, valid_loader):
    combinations = list(itertools.product(epochs, lr_list))
    experiments = []
    for i, combo in enumerate(combinations):
        num_epochs, lr = combo

        print(f"Running Exp {i+1}: num_epoch={num_epochs}, lr={lr}")
        this_model = SmallVGG().to(device)
        criterion = hyper_params['criterion']
        optimizer = hyper_params['optimizer'](this_model.parameters(), lr=lr)
        train_losses, valid_losses = train_and_evaluate(this_model, train_loader, valid_loader, criterion, optimizer, num_epochs)

        experiments.append({
            "num_epochs": num_epochs,
            "lr": lr,
            "train_losses": train_losses,
            "valid_losses": valid_losses,
            "model_state_dict": this_model.state_dict()
        })

        del this_model, criterion, optimizer
    return experiments

exp2_1 = run_exp2_1(candidate_epochs, candidate_lr, exp2_hyperparams, exp2_train_loader, exp2_valid_loader)
time_str = str(time.time()).replace(".","")
torch.save(exp2_1, f"./models/exp2-1_{time_str}.pth")

exp2_1_loaded = torch.load("./models/exp2-1_17305539358378615.pth")
exp2_1_results = get_experiment_results(exp2_1_loaded, test_hyperparam_names=["num_epochs", "lr"], extra_loader=exp2_test_loader)

plot_el(exp2_1_loaded, ["num_epochs", "lr"], n_rows=4, n_cols=4)

plot_cm(exp2_1_results, ["num_epochs", "lr"], n_rows=4, n_cols=4)

exp2_1_accuracies, exp2_1_f1s = plot_pr(exp2_1_results, ["num_epochs", "lr"], n_rows=4, n_cols=4)
print_metrics(exp2_1_accuracies, exp2_1_f1s)

Accuracies:
0.917 0.825 0.336 0.196 0.912 0.865 0.471 0.196 0.907 0.878 0.663 0.196 0.908 0.880 0.713 0.196 

F1 Score Lists:
0.918 0.942 0.947 0.889 0.925 0.913 0.900 0.931 0.855 0.868 | Avg F1=0.909, Std F1=0.029051096513454423
0.811 0.904 0.880 0.774 0.838 0.791 0.745 0.860 0.703 0.731 | Avg F1=0.804, Std F1=0.06335319369501424
0.040 0.570 0.378 0.189 0.273 0.087 0.002 0.237 0.026 0.004 | Avg F1=0.180, Std F1=0.17824165219527116
0.000 0.328 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 | Avg F1=0.033, Std F1=0.09827503131926378
0.914 0.934 0.945 0.882 0.932 0.896 0.878 0.919 0.863 0.875 | Avg F1=0.904, Std F1=0.02715514894076909
0.866 0.930 0.912 0.808 0.881 0.829 0.809 0.899 0.794 0.769 | Avg F1=0.850, Std F1=0.05240811428774516
0.154 0.743 0.509 0.321 0.550 0.430 0.122 0.550 0.005 0.080 | Avg F1=0.347, Std F1=0.23453887474256918
0.000 0.328 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 | Avg F1=0.033, Std F1=0.09827503131926378
0.916 0.926 0.937 0.875 0.922 0.905 0.879 0.919 0.858 0.863 | Avg F1=0.900, Std F1=0.027185167062975517
0.875 0.932 0.913 0.830 0.890 0.849 0.841 0.904 0.810 0.815 | Avg F1=0.866, Std F1=0.04073441033554439
0.596 0.770 0.749 0.632 0.679 0.614 0.618 0.699 0.294 0.526 | Avg F1=0.618, Std F1=0.1283615011641036
0.000 0.328 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 | Avg F1=0.033, Std F1=0.09827503131926378
0.904 0.943 0.926 0.881 0.914 0.913 0.876 0.921 0.863 0.850 | Avg F1=0.899, Std F1=0.02860358424237646
0.879 0.924 0.925 0.821 0.886 0.867 0.832 0.905 0.820 0.820 | Avg F1=0.868, Std F1=0.040548747485917386
0.689 0.842 0.773 0.646 0.711 0.672 0.640 0.763 0.460 0.549 | Avg F1=0.674, Std F1=0.10523037482252583
0.000 0.328 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 | Avg F1=0.033, Std F1=0.09827503131926378
Best: 1-th

plot_rocauc(exp2_1_results, ["num_epochs", "lr"], curve_type="all", n_rows=4, n_cols=4)

plot_rocauc(exp2_1_results, ["num_epochs", "lr"], curve_type="macro_micro", n_rows=4, n_cols=4)

exp2_2_hyperparams = {
    "num_epoch": 15,
    "criterion": nn.CrossEntropyLoss(),
    "transform": A.Compose([
        A.Normalize(mean=exp2_mean, std=exp2_std),
        ToTensorV2()
    ]),
    "optimizer":optim.Adam,
}

# More detailed candidate learning rates around 1e-3, that is 10e-4.
exp2_2_candidate_lr = np.geomspace(1e-4, 5e-3, 8)
print(exp2_2_candidate_lr)

[0.0001     0.00017487 0.00030579 0.00053472 0.00093506 0.00163512
 0.0028593  0.005     ]

def run_exp2_2(lr_list, hyper_params, train_loader, test_loader):
    experiments = []
    for i, lr in enumerate(lr_list):

        print(f"Running Exp {i+1}: lr={lr}")
        this_model = SmallVGG().to(device)
        num_epochs = hyper_params['num_epoch']
        criterion = hyper_params['criterion']
        optimizer = hyper_params['optimizer'](this_model.parameters(), lr=lr)
        train_losses, valid_losses = train_and_evaluate(this_model, train_loader, test_loader, criterion, optimizer, num_epochs)

        experiments.append({
            "num_epochs": num_epochs,
            "lr": lr,
            "train_losses": train_losses,
            "valid_losses": valid_losses,
            "model_state_dict": this_model.state_dict()
        })

        del this_model, criterion, optimizer
    return experiments

exp2_2 = run_exp2_2(exp2_2_candidate_lr, exp2_2_hyperparams, exp2_train_loader, exp2_valid_loader)
time_str = str(time.time()).replace(".", "")
torch.save(exp2_2, f"./models/exp2-2_{time_str}.pth")

exp2_2_loaded = torch.load("./models/exp2-2_17305693866477516.pth")
exp2_2_results = get_experiment_results(exp2_2_loaded, test_hyperparam_names=["num_epochs", "lr"], extra_loader=exp2_test_loader)

plot_el(exp2_2_loaded, ["num_epochs", "lr"], n_rows=2, n_cols=4)

plot_cm(exp2_2_results, ["num_epochs", "lr"], n_rows=2, n_cols=4)

exp2_2_accuracies, exp2_2_f1s = plot_pr(exp2_2_results, ["num_epochs", "lr"], n_rows=2, n_cols=4)
print_metrics(exp2_2_accuracies, exp2_2_f1s)

Accuracies:
0.873 0.885 0.905 0.906 0.912 0.903 0.893 0.882 

F1 Score Lists:
0.873 0.929 0.912 0.822 0.888 0.851 0.829 0.901 0.795 0.798 | Avg F1=0.860, Std F1=0.04542290722465954
0.892 0.936 0.916 0.850 0.905 0.854 0.843 0.906 0.819 0.808 | Avg F1=0.873, Std F1=0.04146430928239567
0.908 0.942 0.941 0.874 0.921 0.888 0.857 0.924 0.840 0.839 | Avg F1=0.893, Std F1=0.03748405274950539
0.911 0.942 0.941 0.868 0.923 0.889 0.875 0.917 0.868 0.829 | Avg F1=0.896, Std F1=0.03485676831682514
0.913 0.936 0.943 0.887 0.928 0.912 0.875 0.926 0.852 0.875 | Avg F1=0.905, Std F1=0.029020749926052154
0.901 0.932 0.937 0.880 0.908 0.912 0.873 0.918 0.835 0.841 | Avg F1=0.894, Std F1=0.033859473636702976
0.887 0.926 0.934 0.862 0.900 0.882 0.854 0.915 0.829 0.836 | Avg F1=0.883, Std F1=0.035070141262265965
0.873 0.921 0.919 0.859 0.888 0.880 0.832 0.891 0.814 0.827 | Avg F1=0.870, Std F1=0.03530344586873121
Best: 5-th

plot_rocauc(exp2_2_results, ["num_epochs", "lr"], curve_type="all", n_rows=2, n_cols=4)

plot_rocauc(exp2_2_results, ["num_epochs", "lr"], curve_type="macro_micro", n_rows=2, n_cols=4)

exp3_universal_train_dataset = SVHNDataset(mat_file=os.path.join(path_dataset, "train_32x32.mat"))

exp3_train_dataset, exp3_valid_dataset = split_train_valid(exp3_universal_train_dataset, train_ratio=0.8)
del exp3_universal_train_dataset

# The mean & std here will only be used for experiment 3-1.
exp3_1_mean, exp3_1_std = exp3_train_dataset.get_meanstd()

exp3_1_hyperparams = {
    "num_epochs": 50,
    "lr": 1e-3,
    "criterion": nn.CrossEntropyLoss(),
    "optimizer": optim.Adam,
    "transform": A.Compose([
        A.Normalize(mean=exp1_mean, std=exp1_std),
        ToTensorV2()
    ])
}

# Transform of train dataset will be altered in the experiments.
exp3_valid_dataset.transform = exp3_1_hyperparams['transform']
exp3_test_dataset = SVHNDataset(mat_file=os.path.join(path_dataset, "test_32x32.mat"), transform=exp3_1_hyperparams["transform"])

print(f"Training Size:{exp3_train_dataset.__len__()}, Validation Size:{exp3_valid_dataset.__len__()}")
print(f"Channel Means: {exp3_1_mean}\nChannel Stds: {exp3_1_std}")

Training Size:58605, Validation Size:14652
Channel Means: [0.43759557604789734, 0.4438314437866211, 0.47290241718292236]
Channel Stds: [0.19781899452209473, 0.2008139044046402, 0.19679588079452515]

# Group 1
candidate_angles = [15, 30, 45, 60]
candidate_crops = [0.08, 0.24, 0.40, 0.60] # Left Boundary

exp3_valid_loader = DataLoader(exp3_valid_dataset, batch_size=128, shuffle=False)
exp3_test_loader = DataLoader(exp3_test_dataset, batch_size=128, shuffle=False)

def run_exp3_1(angles, crops, hyper_params, train_dataset, valid_loader):
    combinations = list(itertools.product(angles, crops))
    experiments = []
    for i, combo in enumerate(combinations):
        angle, crop = combo
        
        print(f"Running Exp {i+1}: angles={angle}, crop={crop}")
        this_model = SmallVGG().to(device)
        num_epochs = hyper_params['num_epochs']
        lr = hyper_params['lr']
        criterion = hyper_params['criterion']
        optimizer = hyper_params['optimizer'](this_model.parameters(), lr=lr)

        # Define Transform
        this_transform = A.Compose([
            A.RandomResizedCrop(32, 32, scale=(crop, 1.0)),
            A.Rotate(limit=angle),
            A.Normalize(mean=exp3_1_mean, std=exp3_1_std),
            ToTensorV2()
        ])

        # Generate Dataset
        print(f"Exp {i+1}: Generating dataset from transform")
        train_dataset.transform = this_transform
        
        train_loader = DataLoader(train_dataset, batch_size=128, shuffle=True)

        # Train Model
        train_losses, valid_losses = train_and_evaluate(this_model, 
                                                       train_loader, 
                                                       valid_loader, 
                                                       criterion, 
                                                       optimizer,
                                                       num_epochs,
                                                       stop_early_params={
                                                           "min_delta": 0.01,
                                                           "patience": 5
                                                       })

        experiments.append({
            "angle": angle,
            "crop": crop,
            "train_losses": train_losses,
            "valid_losses": valid_losses,
            "model_state_dict": this_model.state_dict()
        })

        del this_model, criterion, optimizer
        
        torch.cuda.empty_cache()

    return experiments

exp3_1 = run_exp3_1(candidate_angles, candidate_crops, exp3_1_hyperparams, exp3_train_dataset, exp3_valid_loader)
time_str = str(time.time()).replace(".","")
torch.save(exp3_1, f"./models/exp3-1_{time_str}.pth")

exp3_1_loaded = torch.load("./models/exp3-1_17306646089970913.pth")
exp3_1_results = get_experiment_results(exp3_1_loaded, test_hyperparam_names=["angle", "crop"], extra_loader=exp3_test_loader)

plot_el(exp3_1_loaded, ["angle", "crop"], n_rows=4, n_cols=4)

plot_cm(exp3_1_results, ["angle", "crop"], n_rows=4, n_cols=4)

exp3_accuracies, exp3_f1s = plot_pr(exp3_1_results, ["angle", "crop"], n_rows=4, n_cols=4)
print_metrics(exp3_accuracies, exp3_f1s)

Accuracies:
0.912 0.916 0.917 0.931 0.892 0.898 0.917 0.928 0.886 0.893 0.914 0.928 0.891 0.888 0.910 0.926 

F1 Score Lists:
0.889 0.942 0.942 0.885 0.939 0.922 0.887 0.924 0.843 0.848 | Avg F1=0.902, Std F1=0.03538451976963852
0.895 0.940 0.939 0.894 0.939 0.923 0.894 0.925 0.862 0.864 | Avg F1=0.908, Std F1=0.02847237430596348
0.899 0.936 0.945 0.893 0.935 0.927 0.900 0.923 0.876 0.858 | Avg F1=0.909, Std F1=0.02722603637075925
0.915 0.949 0.952 0.906 0.948 0.937 0.925 0.930 0.892 0.890 | Avg F1=0.925, Std F1=0.02181626856531463
0.892 0.927 0.924 0.845 0.925 0.890 0.870 0.875 0.845 0.837 | Avg F1=0.883, Std F1=0.03265762724812469
0.893 0.934 0.930 0.874 0.925 0.905 0.846 0.901 0.808 0.847 | Avg F1=0.886, Std F1=0.03969946400782543
0.904 0.943 0.947 0.889 0.934 0.921 0.896 0.911 0.873 0.878 | Avg F1=0.910, Std F1=0.024999702435631828
0.918 0.946 0.952 0.903 0.946 0.927 0.908 0.926 0.892 0.900 | Avg F1=0.922, Std F1=0.020141128246637333
0.837 0.922 0.941 0.842 0.913 0.900 0.865 0.915 0.797 0.797 | Avg F1=0.873, Std F1=0.04999486924122103
0.892 0.926 0.933 0.848 0.918 0.893 0.862 0.899 0.829 0.842 | Avg F1=0.884, Std F1=0.035008693882825154
0.901 0.942 0.939 0.871 0.935 0.914 0.901 0.913 0.874 0.877 | Avg F1=0.907, Std F1=0.025555951863289215
0.921 0.944 0.954 0.900 0.950 0.930 0.921 0.922 0.890 0.882 | Avg F1=0.922, Std F1=0.02344422620633198
0.874 0.929 0.931 0.858 0.887 0.893 0.855 0.923 0.823 0.834 | Avg F1=0.881, Std F1=0.036647593322034946
0.857 0.926 0.923 0.847 0.916 0.904 0.882 0.905 0.809 0.796 | Avg F1=0.876, Std F1=0.0446762695421183
0.904 0.940 0.940 0.882 0.923 0.905 0.887 0.914 0.866 0.855 | Avg F1=0.902, Std F1=0.027712221334215276
0.904 0.949 0.955 0.892 0.948 0.912 0.916 0.922 0.893 0.894 | Avg F1=0.919, Std F1=0.023072110295838992
Best: 4-th

plot_rocauc(exp3_1_results, ["angle", "crop"], curve_type="all", n_rows=4, n_cols=4)

plot_rocauc(exp3_1_results, ["angle", "crop"], curve_type="macro_micro", n_rows=4, n_cols=4)

exp3_2_hyperparams = {
    "num_epochs": 50,
    "lr": 1e-3,
    "criterion": nn.CrossEntropyLoss(),
    "optimizer": optim.AdamW,
    "crop":0.6,
    "angle":15,
    "transform": A.Compose([
        A.Normalize(mean=exp1_mean, std=exp1_std),
        ToTensorV2()
    ])
}

class AddBiasTransform:
    def __init__(self, bias: Union[int, Tuple[int, int]]) -> None:
        if isinstance(bias, tuple):
            self.bias1 = bias[0]
            self.bias2 = bias[1]
        else:
            self.bias1 = 0
            self.bias2 = bias

    def __call__(self, img: np.ndarray) -> np.ndarray:
        _dtype = img.dtype
        bias_value = random.randint(self.bias1, self.bias2)
        img = (img.astype(np.int16) + bias_value) % 256
        return img.astype(_dtype)
        
# Group 2
candidate_ratios = [0.25, 0.42, 0.58, 0.75]
candidate_channel_biases = [0, 32, 64, 128]

def run_exp3_2(ratios, biases, hyper_params, train_dataset, valid_dataset):
    combinations = list(itertools.product(ratios, biases))
    experiments = []
    for i, combo in enumerate(combinations):
        ratio, bias = combo
        
        print(f"Running Exp {i+1}: ratio={ratio}, bias={bias}")
        this_model = SmallVGG().to(device)
        num_epochs = hyper_params['num_epochs']
        lr = hyper_params['lr']
        criterion = hyper_params['criterion']
        optimizer = hyper_params['optimizer'](this_model.parameters(), lr=lr)

        # Define Transform
        this_mean, this_std = train_dataset.get_meanstd(bias=bias)
        this_train_transform = A.Compose([
            A.Lambda(image=lambda img, **kwargs: AddBiasTransform(bias)(img)),  # Lambda customized transform block
            A.RandomResizedCrop(32, 32, scale=(hyper_params['crop'], 1.0), ratio=(ratio, 1.0 / ratio)),
            A.Rotate(limit=hyper_params['angle']),
            A.Normalize(mean=this_mean, std=this_std),
            ToTensorV2()
        ])

        this_valid_transform = A.Compose([
            A.Normalize(mean=this_mean, std=this_std),
            ToTensorV2()
        ])

        # Generate Dataset
        print(f"Exp {i+1}: Generating dataset from transform")
        train_dataset.transform = this_train_transform
        valid_dataset.transform = this_valid_transform
        
        train_loader = DataLoader(train_dataset, batch_size=128, shuffle=True)
        valid_loader = DataLoader(valid_dataset, batch_size=128, shuffle=False)

        # Train Model
        train_losses, valid_losses = train_and_evaluate(this_model, 
                                                       train_loader, 
                                                       valid_loader, 
                                                       criterion, 
                                                       optimizer,
                                                       num_epochs,
                                                       stop_early_params={
                                                           "min_delta": 0.01,
                                                           "patience": 5
                                                       })

        experiments.append({
            "ratio": ratio,
            "bias": bias,
            "train_losses": train_losses,
            "valid_losses": valid_losses,
            "model_state_dict": this_model.state_dict()
        })

        del this_model, criterion, optimizer
        # del train_loader, valid_loader
        
        torch.cuda.empty_cache()

    return experiments

exp3_2 = run_exp3_2(candidate_ratios, candidate_channel_biases, exp3_2_hyperparams, exp3_train_dataset, exp3_valid_dataset)
time_str = str(time.time()).replace(".", "")
torch.save(exp3_2, f"./models/exp3-2_{time_str}.pth")

Running Exp 1: ratio=0.25, bias=0

D:\Temps\temp\ipykernel_22900\3883611529.py:17: UserWarning: Using lambda is incompatible with multiprocessing. Consider using regular functions or partial().
  A.Lambda(image=lambda img, **kwargs: AddBiasTransform(bias)(img)),  # Lambda customized transform block

Exp 1: Generating dataset from transform

100%|████████████████████████████████████████████████████████████████████████████████| 458/458 [00:29<00:00, 15.36it/s]

Epoch[1/50], Train Loss:251.9213, Validation Loss:120.0689

100%|████████████████████████████████████████████████████████████████████████████████| 458/458 [00:30<00:00, 15.20it/s]

Epoch[2/50], Train Loss:115.4026, Validation Loss:68.1095

100%|████████████████████████████████████████████████████████████████████████████████| 458/458 [00:29<00:00, 15.36it/s]

Epoch[3/50], Train Loss:83.2680, Validation Loss:53.0528

100%|████████████████████████████████████████████████████████████████████████████████| 458/458 [00:28<00:00, 16.17it/s]

exp3_2_loaded = torch.load("./models/exp3-2_17307070782426171.pth")
exp3_2_results = get_experiment_results(exp3_2_loaded, test_hyperparam_names=["ratio", "bias"], extra_loader=exp3_test_loader)

D:\Temps\temp\ipykernel_22900\2689293624.py:1: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
  exp3_2_loaded = torch.load("./models/exp3-2_17307070782426171.pth")
100%|████████████████████████████████████████████████████████████████████████████████| 204/204 [00:06<00:00, 31.25it/s]

First 10 true labels:
5 2 1 0 6 1 9 1 1 8 ...

First 10 pred labels:
5 2 1 0 6 0 9 1 1 8 ...

First 5 pred_scores:
[1.4084850590734277e-05, 0.00015363673446699977, 4.225009979563765e-05, 0.00040998580516315997, 1.6775904441601597e-05, 0.9974088072776794, 0.0017630067886784673, 3.485668639768846e-05, 8.655283636471722e-06, 0.00014804775128141046] [4.613344373183281e-08, 2.1436735551105812e-05, 0.9987912774085999, 0.0006911041564308107, 1.0980292017848114e-06, 0.0004442851641215384, 2.4895851311157458e-05, 2.0962790586054325e-05, 3.1684444365964737e-06, 1.7334310769001604e-06] [1.404105546498613e-06, 0.9989784955978394, 1.4476317119260784e-05, 2.386712822044501e-06, 1.454915945942048e-05, 7.363043152963655e-08, 1.2108579312553047e-06, 0.000983237405307591, 9.635473929847649e-08, 4.182532393315341e-06] [0.700842559337616, 0.005154207814484835, 0.024481797590851784, 0.005016915500164032, 0.005796228535473347, 0.006218563299626112, 0.0716952309012413, 0.0009235470788553357, 0.09399909526109695, 0.08587180078029633] [0.01026163436472416, 0.0030757756903767586, 0.00025122021907009184, 0.015393401496112347, 0.0022758604027330875, 0.2289666384458542, 0.7372409701347351, 0.0005035883514210582, 0.0014832924352958798, 0.0005476601072587073] ...

100%|████████████████████████████████████████████████████████████████████████████████| 204/204 [00:04<00:00, 41.46it/s]

First 10 true labels:
5 2 1 0 6 1 9 1 1 8 ...

First 10 pred labels:
5 2 1 0 6 0 9 1 1 8 ...

First 5 pred_scores:
[0.0005293998401612043, 0.0007825545035302639, 0.006255303975194693, 0.06350726634263992, 0.0014771796995773911, 0.8754233717918396, 0.04659580811858177, 0.0006909003714099526, 0.0038746146019548178, 0.0008636011625640094] [0.000394436065107584, 0.00019559699285309762, 0.9118548631668091, 0.041615474969148636, 0.0007165743736550212, 0.0009087906801141798, 0.0002736233873292804, 0.0011334215523675084, 0.03999966382980347, 0.0029075529891997576] [0.008978779427707195, 0.7742469310760498, 0.010847742669284344, 0.14756830036640167, 0.0031014459673315287, 0.0008888605516403913, 0.0005936693632975221, 0.03657388314604759, 0.015144744887948036, 0.002055766759440303] [0.8220072388648987, 0.00035255486727692187, 0.010002510622143745, 0.002056727884337306, 2.704974394873716e-05, 6.215804751263931e-05, 0.05037127807736397, 6.38983110547997e-05, 0.11395983397960663, 0.0010967785492539406] [0.014801161363720894, 0.006576502695679665, 0.0033670212142169476, 0.003222749102860689, 0.00635240925475955, 0.026477931067347527, 0.9268736839294434, 0.00794184859842062, 0.0029598709661513567, 0.0014268746599555016] ...

100%|████████████████████████████████████████████████████████████████████████████████| 204/204 [00:05<00:00, 40.72it/s]

First 10 true labels:
5 2 1 0 6 1 9 1 1 8 ...

First 10 pred labels:
5 2 1 0 6 0 9 1 1 8 ...

First 5 pred_scores:
[0.002300459425896406, 0.009195398539304733, 0.015398573130369186, 0.029696986079216003, 0.02971511147916317, 0.5735470652580261, 0.2291260063648224, 0.02381708100438118, 0.06741079688072205, 0.019792525097727776] [1.3163851690478623e-05, 6.1848422774346545e-06, 0.9986153841018677, 0.0005884294514544308, 0.00026158918626606464, 7.750494114588946e-05, 6.78149372106418e-05, 0.00032220143475569785, 4.357180660008453e-05, 4.083500698470743e-06] [0.001602725125849247, 0.9604253768920898, 0.002502964111045003, 0.0034823580645024776, 0.0001377930457238108, 8.330817945534363e-05, 0.005184994079172611, 0.02557222917675972, 0.0008969894261099398, 0.00011132747749798] [0.7898939847946167, 0.0013752433005720377, 0.02061563730239868, 0.0014157386031001806, 0.0004938106285408139, 0.002488094149157405, 0.044465236365795135, 0.00017039837257470936, 0.010004307143390179, 0.12907756865024567] [0.08194509148597717, 0.0077087897807359695, 0.06477726250886917, 0.012072171084582806, 0.005904196295887232, 0.033564288169145584, 0.6243399381637573, 0.1513560712337494, 0.01354144886136055, 0.004790759179741144] ...

100%|████████████████████████████████████████████████████████████████████████████████| 204/204 [00:04<00:00, 43.20it/s]

First 10 true labels:
5 2 1 0 6 1 9 1 1 8 ...

First 10 pred labels:
5 2 1 0 0 0 9 1 1 8 ...

First 5 pred_scores:
[0.0006225058459676802, 0.004992451053112745, 0.015142451971769333, 0.01409939769655466, 0.011914124712347984, 0.9056893587112427, 0.013455850072205067, 0.027691615745425224, 0.0009359631803818047, 0.005456229206174612] [5.2076949941692874e-05, 0.0002324453234905377, 0.9970223307609558, 0.0017034785123541951, 0.00044398437603376806, 0.0002389828150626272, 7.616404036525637e-05, 8.495585643686354e-05, 2.8808593924622983e-05, 0.00011669695959426463] [0.0018049852224066854, 0.925057590007782, 0.0014245978090912104, 0.017692500725388527, 0.031432803720235825, 0.005591353867202997, 0.001352998660877347, 0.001937837223522365, 0.012121611274778843, 0.0015836323145776987] [0.9816752076148987, 0.0008569160127080977, 0.004353683907538652, 0.0002946107997559011, 0.00022513924341183156, 2.658830453583505e-05, 0.0010923953959718347, 5.229896487435326e-05, 0.00887479167431593, 0.0025483472272753716] [0.5703200697898865, 0.017371896654367447, 0.045458607375621796, 0.0011064179707318544, 0.029538340866565704, 0.0006474648253060877, 0.09900231659412384, 0.20791736245155334, 0.022870222106575966, 0.0057672918774187565] ...

100%|████████████████████████████████████████████████████████████████████████████████| 204/204 [00:04<00:00, 44.34it/s]

First 10 true labels:
5 2 1 0 6 1 9 1 1 8 ...

First 10 pred labels:
5 2 1 0 6 0 9 1 1 8 ...

First 5 pred_scores:
[1.495090828029788e-06, 8.52965149533702e-06, 0.0002182294992962852, 0.0010385597124695778, 5.648896149068605e-06, 0.9979828596115112, 0.0005053897621110082, 0.00021224575175438076, 5.496317498909775e-06, 2.1480977011378855e-05] [1.1563573934836313e-05, 4.6002449380466715e-05, 0.9979493021965027, 0.0010337568819522858, 7.79733745730482e-05, 2.0024242985527962e-05, 4.85382042825222e-05, 0.000583082262892276, 0.00022539633209817111, 4.451701443031197e-06] [0.00018858924158848822, 0.9803897738456726, 0.0014331864658743143, 5.1308594265719876e-05, 0.014465335756540298, 0.00015431742940563709, 0.0014340149937197566, 0.00012033357779728249, 0.0016039266483858228, 0.0001591408217791468] [0.8279442191123962, 0.00012675356992986053, 0.005857052747160196, 0.00014356568863149732, 7.45604993426241e-05, 0.00010754881805041805, 0.005312909372150898, 2.2106964024715126e-05, 0.055882491171360016, 0.10452888906002045] [0.12066949158906937, 0.000940801459364593, 0.0015190566191449761, 0.000929150206502527, 2.6033549147541635e-05, 0.0015174942091107368, 0.8709896206855774, 0.002464503748342395, 0.000581133586820215, 0.0003628162376116961] ...

plot_el(exp3_2_loaded, ["ratio", "bias"], n_rows=4, n_cols=4)

plot_cm(exp3_2_results, ["ratio", "bias"], n_rows=4, n_cols=4)

exp3_2_accuracies, exp3_2_f1s = plot_pr(exp3_2_results, ["ratio", "bias"], n_rows=4, n_cols=4)
print_metrics(exp3_2_accuracies, exp3_2_f1s)

Accuracies:
0.924 0.912 0.877 0.885 0.925 0.911 0.913 0.830 0.924 0.913 0.908 0.808 0.928 0.898 0.861 0.852 

F1 Score Lists:
0.916 0.949 0.949 0.902 0.943 0.928 0.892 0.928 0.861 0.883 | Avg F1=0.915, Std F1=0.028178712032797315
0.910 0.938 0.938 0.866 0.932 0.924 0.898 0.914 0.856 0.873 | Avg F1=0.905, Std F1=0.02901513509491587
0.859 0.902 0.923 0.845 0.863 0.896 0.843 0.888 0.809 0.838 | Avg F1=0.867, Std F1=0.03323173297079609
0.864 0.914 0.926 0.856 0.888 0.888 0.854 0.890 0.825 0.844 | Avg F1=0.875, Std F1=0.030212915176477208
0.918 0.941 0.949 0.909 0.945 0.931 0.902 0.921 0.875 0.891 | Avg F1=0.918, Std F1=0.02315118767203162
0.890 0.933 0.936 0.887 0.931 0.924 0.899 0.900 0.854 0.875 | Avg F1=0.903, Std F1=0.02627288161038198
0.905 0.935 0.940 0.881 0.934 0.919 0.890 0.919 0.870 0.873 | Avg F1=0.907, Std F1=0.02514265979976644
0.774 0.862 0.885 0.806 0.838 0.865 0.796 0.820 0.738 0.785 | Avg F1=0.817, Std F1=0.04367929965531259
0.930 0.944 0.947 0.904 0.933 0.922 0.910 0.918 0.884 0.883 | Avg F1=0.917, Std F1=0.021337897218826583
0.883 0.938 0.937 0.884 0.947 0.920 0.898 0.916 0.868 0.855 | Avg F1=0.904, Std F1=0.03000580711454836
0.896 0.932 0.937 0.874 0.930 0.916 0.893 0.906 0.853 0.868 | Avg F1=0.901, Std F1=0.027517225325711638
0.787 0.821 0.879 0.762 0.819 0.814 0.790 0.825 0.720 0.747 | Avg F1=0.796, Std F1=0.04325151448553455
0.929 0.951 0.947 0.908 0.945 0.934 0.907 0.923 0.880 0.889 | Avg F1=0.922, Std F1=0.023251275786566786
0.895 0.931 0.921 0.872 0.907 0.901 0.865 0.901 0.839 0.861 | Avg F1=0.889, Std F1=0.027568983031421735
0.835 0.885 0.919 0.818 0.873 0.882 0.836 0.861 0.766 0.801 | Avg F1=0.848, Std F1=0.04310050816257488
0.809 0.875 0.914 0.806 0.859 0.880 0.842 0.825 0.780 0.813 | Avg F1=0.841, Std F1=0.03902650406749625
Best: 13-th

plot_rocauc(exp3_2_results, ["ratio", "bias"], curve_type="all", n_rows=4, n_cols=4)

plot_rocauc(exp3_2_results, ["ratio", "bias"], curve_type="macro_micro", n_rows=4, n_cols=4)

def exp3_2_peek(dataset, hyper_params, index=21642, ratio=1.0, bias=0):
    exp3_train_dataset_sample = dataset.overwrite(range(exp3_train_dataset.__len__())) # Deep copy of original
    temp_mean, temp_std = exp3_train_dataset_sample.get_meanstd(bias=bias)
    exp3_train_dataset_sample.transform = A.Compose([
                                            A.Lambda(image=lambda img, **kwargs: AddBiasTransform(bias)(img)),  # Lambda customized transform block
                                            A.RandomResizedCrop(32, 32, scale=(hyper_params["crop"], 1.0), ratio=(ratio, 1.0 / ratio)),
                                            A.Rotate(limit=hyper_params["angle"]),
                                            A.Normalize(mean=temp_mean, std=temp_std),
                                            ToTensorV2()
                                        ])
    peek(exp3_train_dataset_sample, index=index)
    del exp3_train_dataset_sample, temp_mean, temp_std

exp3_2_peek(exp3_train_dataset, exp3_2_hyperparams, index=57468, ratio=0.75, bias=0)

D:\Temps\temp\ipykernel_22900\259240058.py:5: UserWarning: Using lambda is incompatible with multiprocessing. Consider using regular functions or partial().
  A.Lambda(image=lambda img, **kwargs: AddBiasTransform(bias)(img)),  # Lambda customized transform block

Peeking data from training set of index 57468.
Image Tnesor Size:torch.Size([3, 32, 32])

exp3_2_peek(exp3_train_dataset, exp3_2_hyperparams, index=57468, ratio=0.75, bias=32)

D:\Temps\temp\ipykernel_22900\259240058.py:5: UserWarning: Using lambda is incompatible with multiprocessing. Consider using regular functions or partial().
  A.Lambda(image=lambda img, **kwargs: AddBiasTransform(bias)(img)),  # Lambda customized transform block

Peeking data from training set of index 57468.
Image Tnesor Size:torch.Size([3, 32, 32])

exp3_2_peek(exp3_train_dataset, exp3_2_hyperparams, index=57468, ratio=0.75, bias=64)

D:\Temps\temp\ipykernel_22900\259240058.py:5: UserWarning: Using lambda is incompatible with multiprocessing. Consider using regular functions or partial().
  A.Lambda(image=lambda img, **kwargs: AddBiasTransform(bias)(img)),  # Lambda customized transform block

Peeking data from training set of index 57468.
Image Tnesor Size:torch.Size([3, 32, 32])

exp3_2_peek(exp3_train_dataset, exp3_2_hyperparams, index=57468, ratio=0.75, bias=128)

D:\Temps\temp\ipykernel_22900\259240058.py:5: UserWarning: Using lambda is incompatible with multiprocessing. Consider using regular functions or partial().
  A.Lambda(image=lambda img, **kwargs: AddBiasTransform(bias)(img)),  # Lambda customized transform block

Peeking data from training set of index 57468.
Image Tnesor Size:torch.Size([3, 32, 32])

CISC3024 Pattern Recognition Final Project¶

Group Members:¶

0. Project Setup¶

0.1 Packages & Device¶

0.2 Global Configurations¶

1. Data Processing and Augmentation¶

1.1 Download Datasets¶

1.1 Notes¶

1.2 Dataset¶

1.2 Peak A Data¶

2. Neural Network¶

2.1 Model Structure¶

2.2 Train and Evaluate Function¶

2.3 Get Predictions¶

2.4 Get Metrics¶

3. Experiments¶

3.0 Preparation¶

3.0.1 Plot Functions¶

Epoch-Loss Curves¶

Get Experiment Results¶

Confusion Matrix¶

Precision-Recall Curve¶

ROC-AUC Curve¶

3.0.1 Datasets¶

3.1 Experiment 1: Optimizer¶

3.2 Experiment 2: Epoch Number and Learning Rate¶

3.2.1 Experiment 2-1: Rough Search¶

3.2.1-1 Epoch-Loss Curve¶

3.2.1-2 Confusion Matrix¶

3.2.1-3 Precision-Recall Curve¶

3.2.2 Experiment 2-2: Detailed¶

3.2.2-1 Epoch-Loss Curve¶

3.2.2-2 Confusion Matrix¶

3.2.2-3 Precision-Recall Curve¶

3.2.2-4 ROC-AUC Curve¶

3.3 Experiment 3: Image Augmentation Parameters¶

3.3.1 Experiment 3-1: Rotation Angles and Crop Ratios¶

3.3.2 Experiment 3-2: Ratios & Bias¶